#读取csv文件part-00001-ada4b76e-d96c-4cb5-bb12-a60b47a276c8-c000.csv，分隔符为'\\x7f\\x5e',engin='python' ，文件格式调整为parquet
import pandas as pd
import os
from sklearn.metrics import roc_curve, roc_auc_score
import numpy as np

os.chdir('557CalKS')
df = pd.read_csv('557全部变量part-00000-106b4f3f-9f81-495b-9510-8eaf4cec6622-c000.csv',sep='\\x7f\\x5e',engine='python')
print(df.head(1))

#所有列名改为大写
df.columns = df.columns.str.upper()
# 定义要分析的字段列表
score_fields = ['SCORE_557', 'QYZXMODEL', 'HNDGMODEL', 'JXJKMODEL', 'FICOMODEL', 'GRZXMODEL', 'HNGRMODEL', 'GSMODEL']

# 确保 APPLY_DT 的格式正确
print(df.head(1))
df['APPLY_DT'] = pd.to_datetime(df['APPLY_DT'])

# 直接使用PFLAG_30D作为Y标签
df['Y'] = df['PFLAG_30D']

# 删除Y标签缺失的记录
df = df.dropna(subset=['Y'])

# 打印Y的分布
print("\nY标签分布：")
print(df['Y'].value_counts())
print(f"Y标签缺失率: {df['Y'].isna().mean():.2%}")

# 计算并打印各模型的KS和AUC
def calculate_metrics(y_true, y_pred):
    fpr, tpr, _ = roc_curve(y_true, y_pred)
    ks = max(abs(tpr - fpr))
    auc = roc_auc_score(y_true, y_pred)
    return ks, auc

print("\n各模型评估指标：")
for field in score_fields:
    if field in df.columns:
        # 去除空值
        valid_mask = ~df[field].isna()
        if valid_mask.any():
            try:
                ks, auc = calculate_metrics(df.loc[valid_mask, 'Y'], df.loc[valid_mask, field])
                print(f"\n{field}评估指标:")
                print(f"样本量: {valid_mask.sum()}")
                print(f"KS值: {ks:.4f}")
                print(f"AUC值: {auc:.4f}")
                print(f"GINI系数: {(2*auc-1):.4f}")
            except Exception as e:
                print(f"{field} 计算指标时出错: {str(e)}")
        else:
            print(f"{field} 全部为空值")

# 打印SCORE_557和Y的前1000行
print("\nSCORE_557和Y的前1000行：")
print(df[['SCORE_557', 'Y']].head(1000))

# 打印缺失值信息
print("\n各模型缺失值数量：")
for field in score_fields:
    if field in df.columns:
        missing_count = df[field].isna().sum()
        total_count = len(df)
        print(f"{field} 缺失值数量: {missing_count}, 缺失率: {missing_count/total_count:.2%}")

